package org.dbrd.preprocessor;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicInteger;

import org.dbrd.model.ISummaryFileFormatConstants;
import org.dbrd.model.PrecessingBugReport;
import org.dbrd.preprocessor.filters.AbstractWordFilter;
import org.dbrd.preprocessor.filters.HTMLSymbolRemover;
import org.dbrd.preprocessor.filters.HTMLTagFilter;
import org.dbrd.preprocessor.filters.LowerCaseWordFilter;
import org.dbrd.preprocessor.filters.NonAlphabetOrNumberSymbolsRemover;
import org.dbrd.preprocessor.filters.StemmingWordFilter;
import org.dbrd.preprocessor.filters.StopWordFilter;
import org.dbrd.preprocessor.filters.TokenizerWordFilter;
import org.dbrd.util.Tracer;

public class Preprocessor {

	// private static final String BUG_REPORT_DIRECTORY = Config.;

	// public static final String SUMMARY_REPORT_PATH =
	// Config.OPENOFFICE_SUMMARY_REPORT_PATH;

	private final int WORKER_COUNT = 60;

	public static final AbstractWordFilter[] SUMMARY_FILTERS;

	public static final AbstractWordFilter[] DESCRIPTION_FILTERS;

	static {
		List<AbstractWordFilter> filters = new ArrayList<AbstractWordFilter>();
		filters.add(new LowerCaseWordFilter());
		filters.add(new TokenizerWordFilter());
		filters.add(new HTMLSymbolRemover());
		filters.add(new StopWordFilter());
		filters.add(new StemmingWordFilter());
		filters.add(new NonAlphabetOrNumberSymbolsRemover());

		SUMMARY_FILTERS = filters
				.toArray(new AbstractWordFilter[filters.size()]);

		filters.clear();
		filters.add(new LowerCaseWordFilter());
		filters.add(new HTMLTagFilter());
		filters.add(new TokenizerWordFilter());
		filters.add(new HTMLSymbolRemover());
		filters.add(new StopWordFilter());
		filters.add(new StemmingWordFilter());
		filters.add(new NonAlphabetOrNumberSymbolsRemover());

		DESCRIPTION_FILTERS = filters.toArray(new AbstractWordFilter[filters
				.size()]);
	}

	/**
	 * @param args
	 */
	// public static void main(String[] args) {
	// String directory = BUG_REPORT_DIRECTORY;
	// new Preprocessor(new File(SUMMARY_REPORT_PATH), new File(directory));
	// }
	private final File target;
	public int CountNegativeDupliacates = 0;

	protected final File reportDirectory;

	private final AbstractWordFilter[] summaryFilters;

	private final AbstractWordFilter[] descrptionFilters;

	private final List<PrecessingBugReport> reports;

	private final Map<Integer, PrecessingBugReport> index;

	private final List<PrecessingBugReport> reportsTemp;

	private final Map<Integer, PrecessingBugReport> indexTemp;

	public int StartIdLimit;

	public int offsetStartId;

	public int startTrainID;
	public int endTrainID;

	private int skipped;

	private AtomicInteger indicator;

	public Preprocessor(File target, File reportDirectory) {
		if (!target.getParentFile().exists()) {
			target.getParentFile().mkdirs();
		}
		this.indicator = new AtomicInteger();
		this.target = target;
		this.reportDirectory = reportDirectory;
		this.reports = new ArrayList<PrecessingBugReport>();
		this.index = new HashMap<Integer, PrecessingBugReport>();
		this.reportsTemp = new ArrayList<PrecessingBugReport>();
		this.indexTemp = new HashMap<Integer, PrecessingBugReport>();
		this.summaryFilters = SUMMARY_FILTERS;
		this.descrptionFilters = DESCRIPTION_FILTERS;
	}

	public int process(int startId, int offestStartId, boolean fixed,
			int startTrainID, int endTrainID) {
		Tracer.sinfo("preprocess bug reports in directory " + reportDirectory);

		StartIdLimit = startId;
		offsetStartId = offestStartId;
		this.startTrainID = startTrainID;
		this.endTrainID = endTrainID;

		readReports();
		processSummaries();
		processDescriptions();
		checkReports();
		buildDuplicateRelations();
		save();

		Tracer.sinfo("");
		Tracer.sinfo("preprocessed " + this.reports.size() + " reports.");
		Tracer.sinfo("preprocessed " + CountNegativeDupliacates
				+ " Negative Duplicate Reports.");

		Tracer.sinfo("===end===");

		if (fixed) {

			return 200;

		} else {
			return CountNegativeDupliacates - 30 + 200;
		}

	}

	private void processDescriptions() {
		Tracer.sinfo("processing descriptions...");
		int indicator = 0;
		for (PrecessingBugReport report : reports) {
			//Tracer.sinfo("processing description of report " + indicator++);
			List<String> words = new LinkedList<String>();
			words.add(report.getOriginalDescription());
			List<String> result = new ArrayList<String>();
			for (AbstractWordFilter filter : this.descrptionFilters) {
				result.clear();
				for (String word : words) {
					result.addAll(Arrays.asList(filter.process(word)));
				}
				words.clear();
				words.addAll(result);
			}
			StringBuilder builder = new StringBuilder();
			for (String word : result) {
				builder.append(word).append(" ");
			}
			report.setProcessedDescription(builder.toString());
		}
	}

	private void processSummaries() {
		Tracer.sinfo("processing summaries...");
		int indicator = 0;
		for (PrecessingBugReport report : reports) {
			indicator++;
			// Tracer.sinfo("processing summary of report " + indicator++);
			List<String> words = new LinkedList<String>();
			words.add(report.getOriginalSummary());
			List<String> result = new ArrayList<String>();
			for (AbstractWordFilter filter : this.summaryFilters) {
				result.clear();
				for (String word : words) {
					result.addAll(Arrays.asList(filter.process(word)));
				}
				words.clear();
				words.addAll(result);
			}
			StringBuilder builder = new StringBuilder();
			for (String word : result) {
				builder.append(word).append(" ");
			}
			report.setProcessedSummary(builder.toString());
		}
	}

	private void save() {
		Tracer.sinfo("saving preprocessed result to " + this.target + "...");
		if (target.getParent() != null) {
			target.getParentFile().mkdirs();
		}
		PrintWriter writer = null;
		try {
			writer = new PrintWriter(new FileWriter(target));
			writer.print(ISummaryFileFormatConstants.REPORT_SEPARATOR);
			writer.print("    ");
			writer.println(ISummaryFileFormatConstants.HEAD_COMMENTS);
			for (PrecessingBugReport report : reports) {
				writer.print(ISummaryFileFormatConstants.KEY_ID);
				writer.print('=');
				writer.println(report.getId());

				// writer.print(ISummaryFileFormatConstants.KEY_ORIGINAL_SUMMARY);
				// writer.print('=');
				// writer.println(report.getOriginalSummary());

				writer.print(ISummaryFileFormatConstants.KEY_PROCESSED_SUMMARY);
				writer.print('=');
				writer.println(report.getProcessedSummary());

				// writer.print(ISummaryFileFormatConstants.KEY_ORIGINAL_DESCRIPTION);
				// writer.print('=');
				// writer.println(report.getOriginalDescription());

				writer.print(ISummaryFileFormatConstants.KEY_PROCESSED_DESCRIPTION);
				writer.print('=');
				writer.println(report.getProcessedDescription());

				writer.print(ISummaryFileFormatConstants.KEY_DUPLICATE_ID);
				writer.print('=');
				writer.println(report.isDuplicated() ? report.getDuplicateId()
						: "");

				writer.print(ISummaryFileFormatConstants.KEY_COMPONENET);
				writer.print('=');
				writer.println(report.getComponent());

				writer.print(ISummaryFileFormatConstants.KEY_SUB_COMPONENT);
				writer.print('=');
				writer.println(report.getSubComponent());

				writer.print(ISummaryFileFormatConstants.KEY_VERSION);
				writer.print('=');
				writer.println(report.getVersion());

				writer.print(ISummaryFileFormatConstants.KEY_PRIORITY);
				writer.print('=');
				writer.println(report.getPriority());

				writer.print(ISummaryFileFormatConstants.KEY_ISSUE_TYPE);
				writer.print('=');
				writer.println(report.getIssueType());

				writer.println(ISummaryFileFormatConstants.REPORT_SEPARATOR);

				// writer.println(ISummaryFileFormatConstants.KEY_ID + "=" +
				// report.getId());
				// writer.println(ISummaryFileFormatConstants.KEY_ORIGINAL_SUMMARY
				// + "=" + report.getOriginalSummary());
				// writer.println(ISummaryFileFormatConstants.KEY_PROCESSED_SUMMARY
				// + "=" + (report.isDuplicated() ? report.getDuplicateId() :
				// ""));
			}
		} catch (IOException e) {
			throw new RuntimeException(e);
		} finally {
			if (writer != null) {
				writer.close();
			}
		}
	}

	private void buildDuplicateRelations() {
		//Tracer.sinfo("building duplicate relations...");
		//Tracer.sinfo("locating duplicate report...");

		int indicator = 1;
		for (PrecessingBugReport report : reports) {
			indicator++;
			// Tracer.sinfo("locate duplicate for report " + indicator++);
			report.locateDuplicate(index);
		}
		Tracer.sinfo("");
		indicator = 1;
		//Tracer.sinfo("pre adjust duplicate id...");
		for (PrecessingBugReport report : reports) {
			indicator++;
			// Tracer.sinfo("pre adjust duplicate id for report: " +
			// indicator++);
			if (report.isDuplicated()) {
				report.preAdjustDuplicateId();
			}
		}
		Tracer.sinfo("");
		indicator = 1;
		//Tracer.sinfo("post adjust duplicate id...");
		for (PrecessingBugReport report : reports) {
			indicator++;
			// Tracer.sinfo("post adjust duplicate id for report: " +
			// indicator++);
			report.postAdjustDuplicateId();
		}
		Tracer.sinfo("");
	}

	private void checkReports() {
		for (Iterator<PrecessingBugReport> iterator = this.reports.iterator(); iterator
				.hasNext();) {
			PrecessingBugReport report = iterator.next();
			int id = report.getId();
			id = id + 0;
			id++;
			if (isEmpty(report.getOriginalDescription())
					|| isEmpty(report.getOriginalSummary())
					|| isEmpty(report.getProcessedDescription())
					|| isEmpty(report.getProcessedSummary())) {
				// Tracer.ssevere("report " + report.getId()
				// + " is empty and will be removed." + ++this.skipped);
				this.index.remove(report.getId());
				iterator.remove();
			}
		}
	}

	private boolean isEmpty(String string) {
		return string == null || string.trim().length() == 0;
	}

	class Worker extends Thread {

		private final ConcurrentLinkedQueue<File> queue;

		public Worker(ConcurrentLinkedQueue<File> queue) {
			super();
			this.queue = queue;
		}

		@Override
		public void run() {
			File file;
			while ((file = queue.poll()) != null) {
				//Tracer.sinfo("reading report " + indicator.incrementAndGet());
				PrecessingBugReport[] reportArray = readReport(file);
				if (reportArray == null) {
					continue;
				}
				for (PrecessingBugReport report : reportArray) {
					if (report.getOriginalDescription() == null
							|| report.getOriginalDescription().length() == 0
							|| report.getOriginalSummary() == null
							|| report.getOriginalSummary().length() == 0) {
						// a report without description or summary is not
						// processed.
						continue;
					}
					synchronized (index) {
						synchronized (reports) {
							index.put(report.getId(), report);
							reports.add(report);

							if (report.getId() >= startTrainID
									&& report.getId() <endTrainID) {

								PrecessingBugReport precessingBugReport = cloneDuplicateReport(report);
								index.put(precessingBugReport.getId(), precessingBugReport);
								reports.add(precessingBugReport);

							}

							// Crazy shit

						}
					}
				}
			}
		}
	}

	private void readReports() {
		File[] reportFiles = getAllReportFiles();

		Tracer.sinfo("reading bug reports... Am here now.");
		// int indicator = 1;
		final ConcurrentLinkedQueue<File> queue = new ConcurrentLinkedQueue<File>();
		for (File file : reportFiles) {
			// Tracer.sinfo("Adding to queue to read: " + " " + file.getName());
			queue.add(file);
		}

		Tracer.sinfo("############### Totol Files Size to Read....  "
				+ queue.size());

		Tracer.sinfo("Startinf new Worker.... ");
		ArrayList<Worker> workers = new ArrayList<Worker>();
		for (int i = 0; i < WORKER_COUNT; i++) {
			workers.add(new Worker(queue));
		}
		for (Worker w : workers) {
			w.start();
		}
		for (Worker w : workers) {
			try {
				w.join();
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
		}
		Collections.sort(this.reports, new Comparator<PrecessingBugReport>() {

			@Override
			public int compare(PrecessingBugReport paramT1,
					PrecessingBugReport paramT2) {
				return paramT1.getId() - paramT2.getId();
			}
		});
		//dupicatesAfter(reports);
		// reports.addAll(reportsTemp);
		// index.putAll(indexTemp);

	}

/*	public void dupicatesAfter(List<PrecessingBugReport> reports) {
		CountNegativeDupliacates = 0;

		for (PrecessingBugReport report : reports) {

			if (report.getId() >= StartIdLimit) {
				System.out.println("Got " + StartIdLimit
						+ ", Duplicates COunt: " + CountNegativeDupliacates);
				break;

			}
			// AfterHowManyGetDuplicates++;
			if (report.isDuplicated()) {

				if (report.getDuplicateId() < StartIdLimit) {
					CountNegativeDupliacates++;
				}
			}
		}
	}*/

	protected File[] getAllReportFiles() {
		return this.reportDirectory.listFiles();
	}

	/**
	 * at present, only [id], [summary], [description],[dupId]are considered.
	 * 
	 * report format.
	 * 
	 * ID:84934 Summary:ODFF: DAYS360 compliance Component:Spreadsheet
	 * SubComponent:programming CreateTime: Wed Jan 2 01:17:00 +0000 2008
	 * Description:description dupID: dupTime:
	 * 
	 * @param reportFile
	 * 
	 * @return
	 */
	protected PrecessingBugReport[] readReport(File reportFile) {
		BufferedReader in = null;

		try {
			in = new BufferedReader(new FileReader(reportFile));
			int id = -1;
			String summary = null;
			String description = null;
			int dupID = -1;
			for (String line = in.readLine(); line != null; line = in
					.readLine()) {
				if (line.startsWith("ID")) {
					id = Integer.parseInt(getContentFromStringLine(line));
				} else if (line.startsWith("Summary")) {
					summary = getContentFromStringLine(line);
				} else if (line.startsWith("Description")) {
					description = getContentFromStringLine(line);
				} else if (line.startsWith("dupID")) {
					dupID = extractDuplicateID(getContentFromStringLine(line));
				}
			}

			if (id < 0 || dupID < 0 || summary == null || description == null) {
				throw new RuntimeException("invalid file format.");
			}
			return new PrecessingBugReport[] { new PrecessingBugReport(id,
					summary, description, dupID, 0, null, null, null, null) };
		} catch (IOException e) {
			e.printStackTrace();
			throw new RuntimeException("cannot read report [" + reportFile
					+ "] due to IO exception.");
		} finally {
			if (in != null) {
				try {
					in.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}
	}

	private String getContentFromStringLine(String line) {
		return line.substring(line.indexOf(":") + 1);
	}

	private int extractDuplicateID(String duplicateID) {
		int start = -1;
		int end = -1;

		int i;
		for (i = 0; i < duplicateID.length(); i++) {
			char c = duplicateID.charAt(i);
			if (c <= '9' && c >= '0') {
				start = i;
				break;
			}
		}
		for (i++; i < duplicateID.length(); i++) {
			char c = duplicateID.charAt(i);
			if (!(c <= '9' && c >= '0')) {
				break;
			}
		}
		end = i;
		if (start != -1) {
			return Integer.parseInt(duplicateID.substring(start, end));
		}
		return PrecessingBugReport.NON_DUPLICATE;
	}

	// public PrecessingBugReport(int id, String originalSummary, String
	// originalDescription, int duplicateId, final int priority,
	// final String version, final String subComponent, final String issueType,
	// String component) {
	// super(id, originalSummary, null, originalDescription, null, duplicateId,
	// priority, version, subComponent, issueType,
	// component);

	private PrecessingBugReport cloneDuplicateReport(
			PrecessingBugReport newReport) {

		// Took-negative value
		// Took-negative value
		PrecessingBugReport clonedReport=null;
		
		if(PrecessingBugReport.NON_DUPLICATE==newReport.getDuplicateId())
		{
		  clonedReport = new PrecessingBugReport(
				-900000+newReport.getId(), newReport.getOriginalSummary(),
				newReport.getOriginalDescription(),newReport.getDuplicateId(), newReport.getPriority(),
				newReport.getVersion(), newReport.getSubComponent(),
				newReport.getIssueType(), newReport.getComponent());
		}else
		{
			  clonedReport = new PrecessingBugReport(
						-900000+newReport.getId(), newReport.getOriginalSummary(),
						newReport.getOriginalDescription(),-900000+newReport.getDuplicateId(), newReport.getPriority(),
						newReport.getVersion(), newReport.getSubComponent(),
						newReport.getIssueType(), newReport.getComponent());
		}

		
		
		//System.err.println("Duplicate ID of "+clonedReport.getDuplicateId());
		return clonedReport;

	}

}
